{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "62d9417e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 1;\n",
       "                var nbb_unformatted_code = \"%load_ext autoreload\\n%autoreload 2\\n%load_ext nb_black\\nimport sys, os\\n\\nsys.path.insert(0, os.path.abspath(\\\"../\\\"))\\nsys.path.insert(0, os.path.abspath(\\\"../../KnowledgeGraphBasic\\\"))\";\n",
       "                var nbb_formatted_code = \"%load_ext autoreload\\n%autoreload 2\\n%load_ext nb_black\\nimport sys, os\\n\\nsys.path.insert(0, os.path.abspath(\\\"../\\\"))\\nsys.path.insert(0, os.path.abspath(\\\"../../KnowledgeGraphBasic\\\"))\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%load_ext nb_black\n",
    "import sys, os\n",
    "\n",
    "sys.path.insert(0, os.path.abspath(\"../\"))\n",
    "sys.path.insert(0, os.path.abspath(\"../../KnowledgeGraphBasic\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "92ba20a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 2;\n",
       "                var nbb_unformatted_code = \"from tqdm import tqdm\\n\\nfrom common_utils import read_json, save_to_json\\nfrom Freebase.classes import FreebaseWrapper\";\n",
       "                var nbb_formatted_code = \"from tqdm import tqdm\\n\\nfrom common_utils import read_json, save_to_json\\nfrom Freebase.classes import FreebaseWrapper\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "from common_utils import read_json, save_to_json\n",
    "from Freebase.classes import FreebaseWrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c953ab9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44337"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 3;\n",
       "                var nbb_unformatted_code = \"trains = read_json(\\\"GrailQA_v1.0/grailqa_v1.0_train.json\\\")\\nlen(trains)\";\n",
       "                var nbb_formatted_code = \"trains = read_json(\\\"GrailQA_v1.0/grailqa_v1.0_train.json\\\")\\nlen(trains)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trains = read_json(\"GrailQA_v1.0/grailqa_v1.0_train.json\")\n",
    "len(trains)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "92af3ffd",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'qid': 3206374001000,\n",
       " 'question': 'the leaders of the earliest established religious organization are given what title?',\n",
       " 'answer': [{'answer_type': 'Entity',\n",
       "   'answer_argument': 'm.05rd8',\n",
       "   'entity_name': 'Pope'}],\n",
       " 'function': 'argmin',\n",
       " 'num_node': 3,\n",
       " 'num_edge': 2,\n",
       " 'graph_query': {'nodes': [{'nid': 0,\n",
       "    'node_type': 'class',\n",
       "    'id': 'religion.religious_leadership_title',\n",
       "    'class': 'religion.religious_leadership_title',\n",
       "    'friendly_name': 'Religious Leadership Title',\n",
       "    'question_node': 1,\n",
       "    'function': 'none'},\n",
       "   {'nid': 1,\n",
       "    'node_type': 'class',\n",
       "    'id': 'religion.religious_organization_leadership',\n",
       "    'class': 'religion.religious_organization_leadership',\n",
       "    'friendly_name': 'Religious Organization Leadership',\n",
       "    'question_node': 0,\n",
       "    'function': 'none'},\n",
       "   {'nid': 2,\n",
       "    'node_type': 'literal',\n",
       "    'id': '0^^http://www.w3.org/2001/XMLSchema#int',\n",
       "    'class': 'type.datetime',\n",
       "    'friendly_name': '0',\n",
       "    'question_node': 0,\n",
       "    'function': 'argmin'}],\n",
       "  'edges': [{'start': 0,\n",
       "    'end': 1,\n",
       "    'relation': 'religion.religious_leadership_title.leaders',\n",
       "    'friendly_name': 'Leaders'},\n",
       "   {'start': 1,\n",
       "    'end': 2,\n",
       "    'relation': 'religion.religious_organization_leadership.start_date',\n",
       "    'friendly_name': 'Start Date'}]},\n",
       " 'sparql_query': 'PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX : <http://rdf.freebase.com/ns/> \\nSELECT (?x0 AS ?value) WHERE {\\nSELECT DISTINCT ?x0  WHERE { \\n?x0 :type.object.type :religion.religious_leadership_title . \\n?x1 :type.object.type :religion.religious_organization_leadership . \\n{\\nSELECT (MIN(?y2) AS ?x2)  WHERE { \\n?y0 :type.object.type :religion.religious_leadership_title . \\n?y1 :type.object.type :religion.religious_organization_leadership . \\n?y0 :religion.religious_leadership_title.leaders ?y1 . \\n?y1 :religion.religious_organization_leadership.start_date ?y2 . \\nFILTER ( ?y0 != ?y1 && ?y0 != ?y2 && ?y1 != ?y2  )\\n}\\n}\\n?x0 :religion.religious_leadership_title.leaders ?x1 . \\n?x1 :religion.religious_organization_leadership.start_date ?x2 . \\nFILTER ( ?x0 != ?x1 && ?x0 != ?x2 && ?x1 != ?x2  )\\n}\\n}',\n",
       " 'domains': ['religion'],\n",
       " 's_expression': '(ARGMIN religion.religious_leadership_title (JOIN religion.religious_leadership_title.leaders religion.religious_organization_leadership.start_date))'}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 23;\n",
       "                var nbb_unformatted_code = \"trains[2]\";\n",
       "                var nbb_formatted_code = \"trains[2]\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trains[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a49cc6b2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX : <http://rdf.freebase.com/ns/> \n",
      "SELECT (?x0 AS ?value) WHERE {\n",
      "SELECT DISTINCT ?x0  WHERE { \n",
      "?x0 :type.object.type :religion.religious_leadership_title . \n",
      "?x1 :type.object.type :religion.religious_organization_leadership . \n",
      "{\n",
      "SELECT (MIN(?y2) AS ?x2)  WHERE { \n",
      "?y0 :type.object.type :religion.religious_leadership_title . \n",
      "?y1 :type.object.type :religion.religious_organization_leadership . \n",
      "?y0 :religion.religious_leadership_title.leaders ?y1 . \n",
      "?y1 :religion.religious_organization_leadership.start_date ?y2 . \n",
      "FILTER ( ?y0 != ?y1 && ?y0 != ?y2 && ?y1 != ?y2  )\n",
      "}\n",
      "}\n",
      "?x0 :religion.religious_leadership_title.leaders ?x1 . \n",
      "?x1 :religion.religious_organization_leadership.start_date ?x2 . \n",
      "FILTER ( ?x0 != ?x1 && ?x0 != ?x2 && ?x1 != ?x2  )\n",
      "}\n",
      "}\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 24;\n",
       "                var nbb_unformatted_code = \"print(trains[2][\\\"sparql_query\\\"])\";\n",
       "                var nbb_formatted_code = \"print(trains[2][\\\"sparql_query\\\"])\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(trains[2][\"sparql_query\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6e899ef",
   "metadata": {},
   "source": [
    "# 验证数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3e685b1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 10;\n",
       "                var nbb_unformatted_code = \"def _parse_res(res):\\n    try:\\n        anslist = [\\n            v[\\\"value\\\"].replace(\\\"http://rdf.freebase.com/ns/\\\", \\\"\\\")\\n            for i in res\\n            for k, v in i.items()\\n        ]\\n    except:\\n        anslist = []\\n    return anslist\";\n",
       "                var nbb_formatted_code = \"def _parse_res(res):\\n    try:\\n        anslist = [\\n            v[\\\"value\\\"].replace(\\\"http://rdf.freebase.com/ns/\\\", \\\"\\\")\\n            for i in res\\n            for k, v in i.items()\\n        ]\\n    except:\\n        anslist = []\\n    return anslist\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def _parse_res(res):\n",
    "    try:\n",
    "        anslist = [\n",
    "            v[\"value\"].replace(\"http://rdf.freebase.com/ns/\", \"\")\n",
    "            for i in res\n",
    "            for k, v in i.items()\n",
    "        ]\n",
    "    except:\n",
    "        anslist = []\n",
    "    return anslist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3bf7990a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['m.0z3xm0m', 'm.0z3xfvs']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 11;\n",
       "                var nbb_unformatted_code = \"xx = [\\n    {\\\"value\\\": {\\\"type\\\": \\\"uri\\\", \\\"value\\\": \\\"http://rdf.freebase.com/ns/m.0z3xm0m\\\"}},\\n    {\\\"value\\\": {\\\"type\\\": \\\"uri\\\", \\\"value\\\": \\\"http://rdf.freebase.com/ns/m.0z3xfvs\\\"}},\\n]\\n_parse_res(xx)\";\n",
       "                var nbb_formatted_code = \"xx = [\\n    {\\\"value\\\": {\\\"type\\\": \\\"uri\\\", \\\"value\\\": \\\"http://rdf.freebase.com/ns/m.0z3xm0m\\\"}},\\n    {\\\"value\\\": {\\\"type\\\": \\\"uri\\\", \\\"value\\\": \\\"http://rdf.freebase.com/ns/m.0z3xfvs\\\"}},\\n]\\n_parse_res(xx)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "xx = [\n",
    "    {\"value\": {\"type\": \"uri\", \"value\": \"http://rdf.freebase.com/ns/m.0z3xm0m\"}},\n",
    "    {\"value\": {\"type\": \"uri\", \"value\": \"http://rdf.freebase.com/ns/m.0z3xfvs\"}},\n",
    "]\n",
    "_parse_res(xx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1b82868",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [],
   "source": [
    "# lan 版本\n",
    "fb = FreebaseWrapper(end_point=\"http://192.168.4.194:8898/sparql\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4de46cdf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 12;\n",
       "                var nbb_unformatted_code = \"# \\u5b98\\u7f51 \\u7248\\u672c\\nfb = FreebaseWrapper(end_point=\\\"http://192.168.4.194:8899/sparql\\\")\";\n",
       "                var nbb_formatted_code = \"# \\u5b98\\u7f51 \\u7248\\u672c\\nfb = FreebaseWrapper(end_point=\\\"http://192.168.4.194:8899/sparql\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 官网 版本\n",
    "fb = FreebaseWrapper(end_point=\"http://192.168.4.194:8899/sparql\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "24dd0574",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 44337/44337 [19:51<00:00, 37.23it/s]  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "44337"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 13;\n",
       "                var nbb_unformatted_code = \"preds = []\\nfor item in tqdm(trains[:]):\\n    sparql = item[\\\"sparql_query\\\"]\\n    ans = _parse_res(fb.query(sparql))\\n    preds.append(ans)\\nlen(preds)\";\n",
       "                var nbb_formatted_code = \"preds = []\\nfor item in tqdm(trains[:]):\\n    sparql = item[\\\"sparql_query\\\"]\\n    ans = _parse_res(fb.query(sparql))\\n    preds.append(ans)\\nlen(preds)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "preds = []\n",
    "for item in tqdm(trains[:]):\n",
    "    sparql = item[\"sparql_query\"]\n",
    "    ans = _parse_res(fb.query(sparql))\n",
    "    preds.append(ans)\n",
    "len(preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eabb78a5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63e9257e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8d4c545f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6763"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 14;\n",
       "                var nbb_unformatted_code = \"devs = read_json(\\\"GrailQA_v1.0/grailqa_v1.0_dev.json\\\")\\nlen(devs)\";\n",
       "                var nbb_formatted_code = \"devs = read_json(\\\"GrailQA_v1.0/grailqa_v1.0_dev.json\\\")\\nlen(devs)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "devs = read_json(\"GrailQA_v1.0/grailqa_v1.0_dev.json\")\n",
    "len(devs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0ae8ec02",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6763/6763 [01:37<00:00, 69.18it/s] \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "6763"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 15;\n",
       "                var nbb_unformatted_code = \"preds2 = []\\nfor item in tqdm(devs[:]):\\n    sparql = item[\\\"sparql_query\\\"]\\n    ans = _parse_res(fb.query(sparql))\\n    preds2.append(ans)\\nlen(preds2)\";\n",
       "                var nbb_formatted_code = \"preds2 = []\\nfor item in tqdm(devs[:]):\\n    sparql = item[\\\"sparql_query\\\"]\\n    ans = _parse_res(fb.query(sparql))\\n    preds2.append(ans)\\nlen(preds2)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "preds2 = []\n",
    "for item in tqdm(devs[:]):\n",
    "    sparql = item[\"sparql_query\"]\n",
    "    ans = _parse_res(fb.query(sparql))\n",
    "    preds2.append(ans)\n",
    "len(preds2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "187505ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 16;\n",
       "                var nbb_unformatted_code = \"def _get_ans(train_item):\\n    ans = [i[\\\"answer_argument\\\"] for i in item[\\\"answer\\\"]]\\n    return ans\\n\\n\\ndef _cal_precision(preds, goldens):\\n    tp = len(set(preds) & set(goldens))\\n    return tp / (len(preds) + 1e-9)\\n\\n\\ndef _cal_recall(preds, goldens):\\n    tp = len(set(preds) & set(goldens))\\n    return tp / (len(goldens) + 1e-9)\\n\\n\\ndef _cal_PRF1(preds, goldens):\\n    precision = _cal_precision(preds, goldens)\\n    recall = _cal_recall(preds, goldens)\\n    f1 = 2 * (precision * recall) / (precision + recall + 1e-9)\\n    return precision, recall, f1\\n\\n\\ndef _cal_PRF1_average(prf1s):\\n    \\\"\\\"\\\"\\n    \\u8ba1\\u7b97\\u5e73\\u5747\\n    \\\"\\\"\\\"\\n    ave_pre = sum([i[0] for i in prf1s]) / len(prf1s)\\n    ave_rec = sum([i[1] for i in prf1s]) / len(prf1s)\\n    ave_f1 = sum([i[2] for i in prf1s]) / len(prf1s)\\n    return (\\n        ave_pre,\\n        ave_rec,\\n        ave_f1,\\n    )\";\n",
       "                var nbb_formatted_code = \"def _get_ans(train_item):\\n    ans = [i[\\\"answer_argument\\\"] for i in item[\\\"answer\\\"]]\\n    return ans\\n\\n\\ndef _cal_precision(preds, goldens):\\n    tp = len(set(preds) & set(goldens))\\n    return tp / (len(preds) + 1e-9)\\n\\n\\ndef _cal_recall(preds, goldens):\\n    tp = len(set(preds) & set(goldens))\\n    return tp / (len(goldens) + 1e-9)\\n\\n\\ndef _cal_PRF1(preds, goldens):\\n    precision = _cal_precision(preds, goldens)\\n    recall = _cal_recall(preds, goldens)\\n    f1 = 2 * (precision * recall) / (precision + recall + 1e-9)\\n    return precision, recall, f1\\n\\n\\ndef _cal_PRF1_average(prf1s):\\n    \\\"\\\"\\\"\\n    \\u8ba1\\u7b97\\u5e73\\u5747\\n    \\\"\\\"\\\"\\n    ave_pre = sum([i[0] for i in prf1s]) / len(prf1s)\\n    ave_rec = sum([i[1] for i in prf1s]) / len(prf1s)\\n    ave_f1 = sum([i[2] for i in prf1s]) / len(prf1s)\\n    return (\\n        ave_pre,\\n        ave_rec,\\n        ave_f1,\\n    )\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def _get_ans(train_item):\n",
    "    ans = [i[\"answer_argument\"] for i in item[\"answer\"]]\n",
    "    return ans\n",
    "\n",
    "\n",
    "def _cal_precision(preds, goldens):\n",
    "    tp = len(set(preds) & set(goldens))\n",
    "    return tp / (len(preds) + 1e-9)\n",
    "\n",
    "\n",
    "def _cal_recall(preds, goldens):\n",
    "    tp = len(set(preds) & set(goldens))\n",
    "    return tp / (len(goldens) + 1e-9)\n",
    "\n",
    "\n",
    "def _cal_PRF1(preds, goldens):\n",
    "    precision = _cal_precision(preds, goldens)\n",
    "    recall = _cal_recall(preds, goldens)\n",
    "    f1 = 2 * (precision * recall) / (precision + recall + 1e-9)\n",
    "    return precision, recall, f1\n",
    "\n",
    "\n",
    "def _cal_PRF1_average(prf1s):\n",
    "    \"\"\"\n",
    "    计算平均\n",
    "    \"\"\"\n",
    "    ave_pre = sum([i[0] for i in prf1s]) / len(prf1s)\n",
    "    ave_rec = sum([i[1] for i in prf1s]) / len(prf1s)\n",
    "    ave_f1 = sum([i[2] for i in prf1s]) / len(prf1s)\n",
    "    return (\n",
    "        ave_pre,\n",
    "        ave_rec,\n",
    "        ave_f1,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3b4e586e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0, 0.0, 0.0)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 17;\n",
       "                var nbb_unformatted_code = \"_cal_PRF1(_get_ans(trains[0]), preds[0])\";\n",
       "                var nbb_formatted_code = \"_cal_PRF1(_get_ans(trains[0]), preds[0])\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_cal_PRF1(_get_ans(trains[0]), preds[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03b32e52",
   "metadata": {},
   "source": [
    "## freebase-lan版本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "62679b65",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train average p r f1:  (0.8553805024749562, 0.8633598253787897, 0.8570569916090065)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 60;\n",
       "                var nbb_unformatted_code = \"prf1s = []\\nfor pred,item in zip(preds,trains):\\n    golden = _get_ans(item)\\n    p,r,f1 = _cal_PRF1(pred, golden)\\n    prf1s.append((p,r,f1))\\n\\nprf1s_ave = _cal_PRF1_average(prf1s)\\nprint(\\\"train average p r f1: \\\",prf1s_ave)\";\n",
       "                var nbb_formatted_code = \"prf1s = []\\nfor pred, item in zip(preds, trains):\\n    golden = _get_ans(item)\\n    p, r, f1 = _cal_PRF1(pred, golden)\\n    prf1s.append((p, r, f1))\\n\\nprf1s_ave = _cal_PRF1_average(prf1s)\\nprint(\\\"train average p r f1: \\\", prf1s_ave)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prf1s = []\n",
    "for pred, item in zip(preds, trains):\n",
    "    golden = _get_ans(item)\n",
    "    p, r, f1 = _cal_PRF1(pred, golden)\n",
    "    prf1s.append((p, r, f1))\n",
    "\n",
    "prf1s_ave = _cal_PRF1_average(prf1s)\n",
    "print(\"train average p r f1: \", prf1s_ave)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2db3d39e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "56f34e1f",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev average p r f1:  (0.8453256227882852, 0.8550021588978347, 0.8474686866942301)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 63;\n",
       "                var nbb_unformatted_code = \"prf1s = []\\nfor pred, item in zip(preds2, devs):\\n    golden = _get_ans(item)\\n    p, r, f1 = _cal_PRF1(pred, golden)\\n    prf1s.append((p, r, f1))\\n\\nprf1s_ave = _cal_PRF1_average(prf1s)\\nprint(\\\"dev average p r f1: \\\", prf1s_ave)\";\n",
       "                var nbb_formatted_code = \"prf1s = []\\nfor pred, item in zip(preds2, devs):\\n    golden = _get_ans(item)\\n    p, r, f1 = _cal_PRF1(pred, golden)\\n    prf1s.append((p, r, f1))\\n\\nprf1s_ave = _cal_PRF1_average(prf1s)\\nprint(\\\"dev average p r f1: \\\", prf1s_ave)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prf1s = []\n",
    "for pred, item in zip(preds2, devs):\n",
    "    golden = _get_ans(item)\n",
    "    p, r, f1 = _cal_PRF1(pred, golden)\n",
    "    prf1s.append((p, r, f1))\n",
    "\n",
    "prf1s_ave = _cal_PRF1_average(prf1s)\n",
    "print(\"dev average p r f1: \", prf1s_ave)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60e2d9ca",
   "metadata": {},
   "source": [
    "## freebase-官网版本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "728d98e6",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train average p r f1:  (0.9999999992355402, 0.9999999992355402, 0.999999998735639)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 18;\n",
       "                var nbb_unformatted_code = \"prf1s = []\\nfor pred, item in zip(preds, trains):\\n    golden = _get_ans(item)\\n    p, r, f1 = _cal_PRF1(pred, golden)\\n    prf1s.append((p, r, f1))\\n\\nprf1s_ave = _cal_PRF1_average(prf1s)\\nprint(\\\"train average p r f1: \\\", prf1s_ave)\";\n",
       "                var nbb_formatted_code = \"prf1s = []\\nfor pred, item in zip(preds, trains):\\n    golden = _get_ans(item)\\n    p, r, f1 = _cal_PRF1(pred, golden)\\n    prf1s.append((p, r, f1))\\n\\nprf1s_ave = _cal_PRF1_average(prf1s)\\nprint(\\\"train average p r f1: \\\", prf1s_ave)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prf1s = []\n",
    "for pred, item in zip(preds, trains):\n",
    "    golden = _get_ans(item)\n",
    "    p, r, f1 = _cal_PRF1(pred, golden)\n",
    "    prf1s.append((p, r, f1))\n",
    "\n",
    "prf1s_ave = _cal_PRF1_average(prf1s)\n",
    "print(\"train average p r f1: \", prf1s_ave)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a3db25e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "dd453f3b",
   "metadata": {
    "deletable": false,
    "editable": false,
    "run_control": {
     "frozen": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev average p r f1:  (0.9999999992208647, 0.9999999992208647, 0.9999999987210129)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 19;\n",
       "                var nbb_unformatted_code = \"prf1s = []\\nfor pred, item in zip(preds2, devs):\\n    golden = _get_ans(item)\\n    p, r, f1 = _cal_PRF1(pred, golden)\\n    prf1s.append((p, r, f1))\\n\\nprf1s_ave = _cal_PRF1_average(prf1s)\\nprint(\\\"dev average p r f1: \\\", prf1s_ave)\";\n",
       "                var nbb_formatted_code = \"prf1s = []\\nfor pred, item in zip(preds2, devs):\\n    golden = _get_ans(item)\\n    p, r, f1 = _cal_PRF1(pred, golden)\\n    prf1s.append((p, r, f1))\\n\\nprf1s_ave = _cal_PRF1_average(prf1s)\\nprint(\\\"dev average p r f1: \\\", prf1s_ave)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prf1s = []\n",
    "for pred, item in zip(preds2, devs):\n",
    "    golden = _get_ans(item)\n",
    "    p, r, f1 = _cal_PRF1(pred, golden)\n",
    "    prf1s.append((p, r, f1))\n",
    "\n",
    "prf1s_ave = _cal_PRF1_average(prf1s)\n",
    "print(\"dev average p r f1: \", prf1s_ave)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebde3699",
   "metadata": {},
   "source": [
    "# function字段"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "951a67d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 21;\n",
       "                var nbb_unformatted_code = \"from pprint import pprint\";\n",
       "                var nbb_formatted_code = \"from pprint import pprint\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a230bc50",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "{'answer': [{'answer_argument': 'm.05rd8',\n",
      "             'answer_type': 'Entity',\n",
      "             'entity_name': 'Pope'}],\n",
      " 'domains': ['religion'],\n",
      " 'function': 'argmin',\n",
      " 'graph_query': {'edges': [{'end': 1,\n",
      "                            'friendly_name': 'Leaders',\n",
      "                            'relation': 'religion.religious_leadership_title.leaders',\n",
      "                            'start': 0},\n",
      "                           {'end': 2,\n",
      "                            'friendly_name': 'Start Date',\n",
      "                            'relation': 'religion.religious_organization_leadership.start_date',\n",
      "                            'start': 1}],\n",
      "                 'nodes': [{'class': 'religion.religious_leadership_title',\n",
      "                            'friendly_name': 'Religious Leadership Title',\n",
      "                            'function': 'none',\n",
      "                            'id': 'religion.religious_leadership_title',\n",
      "                            'nid': 0,\n",
      "                            'node_type': 'class',\n",
      "                            'question_node': 1},\n",
      "                           {'class': 'religion.religious_organization_leadership',\n",
      "                            'friendly_name': 'Religious Organization '\n",
      "                                             'Leadership',\n",
      "                            'function': 'none',\n",
      "                            'id': 'religion.religious_organization_leadership',\n",
      "                            'nid': 1,\n",
      "                            'node_type': 'class',\n",
      "                            'question_node': 0},\n",
      "                           {'class': 'type.datetime',\n",
      "                            'friendly_name': '0',\n",
      "                            'function': 'argmin',\n",
      "                            'id': '0^^http://www.w3.org/2001/XMLSchema#int',\n",
      "                            'nid': 2,\n",
      "                            'node_type': 'literal',\n",
      "                            'question_node': 0}]},\n",
      " 'num_edge': 2,\n",
      " 'num_node': 3,\n",
      " 'qid': 3206374001000,\n",
      " 'question': 'the leaders of the earliest established religious organization '\n",
      "             'are given what title?',\n",
      " 's_expression': '(ARGMIN religion.religious_leadership_title (JOIN '\n",
      "                 'religion.religious_leadership_title.leaders '\n",
      "                 'religion.religious_organization_leadership.start_date))',\n",
      " 'sparql_query': 'PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> '\n",
      "                 'PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX '\n",
      "                 ': <http://rdf.freebase.com/ns/> \\n'\n",
      "                 'SELECT (?x0 AS ?value) WHERE {\\n'\n",
      "                 'SELECT DISTINCT ?x0  WHERE { \\n'\n",
      "                 '?x0 :type.object.type :religion.religious_leadership_title '\n",
      "                 '. \\n'\n",
      "                 '?x1 :type.object.type '\n",
      "                 ':religion.religious_organization_leadership . \\n'\n",
      "                 '{\\n'\n",
      "                 'SELECT (MIN(?y2) AS ?x2)  WHERE { \\n'\n",
      "                 '?y0 :type.object.type :religion.religious_leadership_title '\n",
      "                 '. \\n'\n",
      "                 '?y1 :type.object.type '\n",
      "                 ':religion.religious_organization_leadership . \\n'\n",
      "                 '?y0 :religion.religious_leadership_title.leaders ?y1 . \\n'\n",
      "                 '?y1 :religion.religious_organization_leadership.start_date '\n",
      "                 '?y2 . \\n'\n",
      "                 'FILTER ( ?y0 != ?y1 && ?y0 != ?y2 && ?y1 != ?y2  )\\n'\n",
      "                 '}\\n'\n",
      "                 '}\\n'\n",
      "                 '?x0 :religion.religious_leadership_title.leaders ?x1 . \\n'\n",
      "                 '?x1 :religion.religious_organization_leadership.start_date '\n",
      "                 '?x2 . \\n'\n",
      "                 'FILTER ( ?x0 != ?x1 && ?x0 != ?x2 && ?x1 != ?x2  )\\n'\n",
      "                 '}\\n'\n",
      "                 '}'}\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 22;\n",
       "                var nbb_unformatted_code = \"for index, item in enumerate(trains):\\n    if item[\\\"function\\\"] != \\\"none\\\":\\n        print(index)\\n        pprint(item)\\n        break\";\n",
       "                var nbb_formatted_code = \"for index, item in enumerate(trains):\\n    if item[\\\"function\\\"] != \\\"none\\\":\\n        print(index)\\n        pprint(item)\\n        break\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for index, item in enumerate(trains):\n",
    "    if item[\"function\"] != \"none\":\n",
    "        print(index)\n",
    "        pprint(item)\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57f29709",
   "metadata": {},
   "source": [
    "# domain字段"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "e73cb457",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 74;\n",
       "                var nbb_unformatted_code = \"from collections import Counter\";\n",
       "                var nbb_formatted_code = \"from collections import Counter\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "aca77c14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "76"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 80;\n",
       "                var nbb_unformatted_code = \"# train\\ndomains = []\\nfor item in trains:\\n    domains.extend(item[\\\"domains\\\"])\\ndomains_c1 = Counter(domains)\\nlen(domains_c1)\";\n",
       "                var nbb_formatted_code = \"# train\\ndomains = []\\nfor item in trains:\\n    domains.extend(item[\\\"domains\\\"])\\ndomains_c1 = Counter(domains)\\nlen(domains_c1)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# train\n",
    "domains = []\n",
    "for item in trains:\n",
    "    domains.extend(item[\"domains\"])\n",
    "domains_c1 = Counter(domains)\n",
    "len(domains_c1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "6fac71da",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'medicine': 2028,\n",
       "         'education': 1149,\n",
       "         'religion': 911,\n",
       "         'user.patrick.default_domain': 19,\n",
       "         'language': 371,\n",
       "         'food': 1294,\n",
       "         'book': 2237,\n",
       "         'music': 2340,\n",
       "         'astronomy': 1560,\n",
       "         'people': 1821,\n",
       "         'architecture': 824,\n",
       "         'fictional_universe': 2253,\n",
       "         'business': 978,\n",
       "         'visual_art': 548,\n",
       "         'comic_books': 1129,\n",
       "         'computer': 1923,\n",
       "         'media_common': 820,\n",
       "         'biology': 1273,\n",
       "         'meteorology': 739,\n",
       "         'olympics': 505,\n",
       "         'tv': 1412,\n",
       "         'aviation': 942,\n",
       "         'travel': 507,\n",
       "         'organization': 531,\n",
       "         'theater': 557,\n",
       "         'spaceflight': 1387,\n",
       "         'basketball': 153,\n",
       "         'geography': 649,\n",
       "         'exhibitions': 305,\n",
       "         'internet': 495,\n",
       "         'automotive': 773,\n",
       "         'cricket': 453,\n",
       "         'interests': 212,\n",
       "         'freebase': 149,\n",
       "         'tennis': 213,\n",
       "         'engineering': 417,\n",
       "         'ice_hockey': 175,\n",
       "         'wine': 427,\n",
       "         'soccer': 388,\n",
       "         'base.lightweight': 125,\n",
       "         'time': 963,\n",
       "         'broadcast': 739,\n",
       "         'library': 42,\n",
       "         'martial_arts': 172,\n",
       "         'digicams': 716,\n",
       "         'opera': 403,\n",
       "         'law': 661,\n",
       "         'government': 1262,\n",
       "         'boxing': 57,\n",
       "         'sports': 1438,\n",
       "         'transportation': 149,\n",
       "         'comic_strips': 87,\n",
       "         'protected_sites': 177,\n",
       "         'distilled_spirits': 334,\n",
       "         'physics': 114,\n",
       "         'dining': 232,\n",
       "         'amusement_parks': 726,\n",
       "         'celebrities': 73,\n",
       "         'military': 271,\n",
       "         'royalty': 452,\n",
       "         'boats': 464,\n",
       "         'projects': 125,\n",
       "         'periodicals': 47,\n",
       "         'symbols': 107,\n",
       "         'common': 108,\n",
       "         'fashion': 68,\n",
       "         'zoos': 129,\n",
       "         'influence': 29,\n",
       "         'user.jonathanwlowe.location': 9,\n",
       "         'chess': 54,\n",
       "         'base.exoplanetology': 127,\n",
       "         'geology': 74,\n",
       "         'skiing': 183,\n",
       "         'bicycles': 87,\n",
       "         'type': 14,\n",
       "         'comedy': 5})"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 81;\n",
       "                var nbb_unformatted_code = \"domains_c1\";\n",
       "                var nbb_formatted_code = \"domains_c1\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "domains_c1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a479720c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "c3bdbc8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "77"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 82;\n",
       "                var nbb_unformatted_code = \"# dev\\ndomains = []\\nfor item in devs:\\n    domains.extend(item[\\\"domains\\\"])\\ndomains_c2 = Counter(domains)\\nlen(domains_c2)\";\n",
       "                var nbb_formatted_code = \"# dev\\ndomains = []\\nfor item in devs:\\n    domains.extend(item[\\\"domains\\\"])\\ndomains_c2 = Counter(domains)\\nlen(domains_c2)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dev\n",
    "domains = []\n",
    "for item in devs:\n",
    "    domains.extend(item[\"domains\"])\n",
    "domains_c2 = Counter(domains)\n",
    "len(domains_c2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "ebff6a0a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'opera': 12,\n",
       "         'rail': 341,\n",
       "         'measurement_unit': 1565,\n",
       "         'cvg': 1248,\n",
       "         'religion': 123,\n",
       "         'conferences': 281,\n",
       "         'business': 55,\n",
       "         'government': 46,\n",
       "         'radio': 221,\n",
       "         'medicine': 178,\n",
       "         'broadcast': 148,\n",
       "         'time': 86,\n",
       "         'exhibitions': 42,\n",
       "         'computer': 219,\n",
       "         'astronomy': 100,\n",
       "         'geography': 44,\n",
       "         'sports': 79,\n",
       "         'soccer': 12,\n",
       "         'book': 123,\n",
       "         'education': 80,\n",
       "         'people': 223,\n",
       "         'tv': 57,\n",
       "         'fictional_universe': 188,\n",
       "         'biology': 126,\n",
       "         'food': 66,\n",
       "         'wine': 42,\n",
       "         'language': 12,\n",
       "         'media_common': 76,\n",
       "         'spaceflight': 87,\n",
       "         'theater': 32,\n",
       "         'organization': 45,\n",
       "         'architecture': 48,\n",
       "         'music': 198,\n",
       "         'military': 8,\n",
       "         'law': 33,\n",
       "         'automotive': 38,\n",
       "         'physics': 22,\n",
       "         'interests': 27,\n",
       "         'type': 6,\n",
       "         'meteorology': 33,\n",
       "         'boats': 60,\n",
       "         'zoos': 7,\n",
       "         'amusement_parks': 50,\n",
       "         'symbols': 9,\n",
       "         'internet': 81,\n",
       "         'basketball': 5,\n",
       "         'olympics': 24,\n",
       "         'common': 4,\n",
       "         'digicams': 53,\n",
       "         'royalty': 18,\n",
       "         'travel': 37,\n",
       "         'comic_books': 48,\n",
       "         'projects': 6,\n",
       "         'aviation': 42,\n",
       "         'protected_sites': 3,\n",
       "         'dining': 11,\n",
       "         'engineering': 17,\n",
       "         'skiing': 10,\n",
       "         'distilled_spirits': 7,\n",
       "         'martial_arts': 11,\n",
       "         'cricket': 16,\n",
       "         'base.lightweight': 12,\n",
       "         'freebase': 5,\n",
       "         'comic_strips': 3,\n",
       "         'visual_art': 18,\n",
       "         'ice_hockey': 6,\n",
       "         'user.jonathanwlowe.location': 1,\n",
       "         'geology': 2,\n",
       "         'periodicals': 1,\n",
       "         'base.exoplanetology': 2,\n",
       "         'transportation': 3,\n",
       "         'tennis': 10,\n",
       "         'library': 2,\n",
       "         'celebrities': 3,\n",
       "         'user.patrick.default_domain': 1,\n",
       "         'bicycles': 1,\n",
       "         'boxing': 1})"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 83;\n",
       "                var nbb_unformatted_code = \"domains_c2\";\n",
       "                var nbb_formatted_code = \"domains_c2\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "domains_c2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "d701c721",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'conferences', 'cvg', 'measurement_unit', 'radio', 'rail'}"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 85;\n",
       "                var nbb_unformatted_code = \"set(domains_c2.keys()) - (set(domains_c1.keys()) & set(domains_c2.keys()))\";\n",
       "                var nbb_formatted_code = \"set(domains_c2.keys()) - (set(domains_c1.keys()) & set(domains_c2.keys()))\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "set(domains_c2.keys()) - (set(domains_c1.keys()) & set(domains_c2.keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bfc6988",
   "metadata": {},
   "source": [
    "# dev数据集 level字段"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "750ddeda",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'qid': 3202959008000,\n",
       " 'question': 'what is the role of opera designer gig who designed the telephone / the medium?',\n",
       " 'answer': [{'answer_type': 'Entity',\n",
       "   'answer_argument': 'm.0b787yg',\n",
       "   'entity_name': 'Set Designer'}],\n",
       " 'function': 'none',\n",
       " 'num_node': 3,\n",
       " 'num_edge': 2,\n",
       " 'graph_query': {'nodes': [{'nid': 0,\n",
       "    'node_type': 'class',\n",
       "    'id': 'opera.opera_designer_role',\n",
       "    'class': 'opera.opera_designer_role',\n",
       "    'friendly_name': 'Opera Designer Role',\n",
       "    'question_node': 1,\n",
       "    'function': 'none'},\n",
       "   {'nid': 1,\n",
       "    'node_type': 'class',\n",
       "    'id': 'opera.opera_designer_gig',\n",
       "    'class': 'opera.opera_designer_gig',\n",
       "    'friendly_name': 'Opera Designer Gig',\n",
       "    'question_node': 0,\n",
       "    'function': 'none'},\n",
       "   {'nid': 2,\n",
       "    'node_type': 'entity',\n",
       "    'id': 'm.0pm2fgf',\n",
       "    'class': 'opera.opera_production',\n",
       "    'friendly_name': 'The Telephone / The Medium',\n",
       "    'question_node': 0,\n",
       "    'function': 'none'}],\n",
       "  'edges': [{'start': 1,\n",
       "    'end': 0,\n",
       "    'relation': 'opera.opera_designer_gig.design_role',\n",
       "    'friendly_name': 'Design Role'},\n",
       "   {'start': 2,\n",
       "    'end': 1,\n",
       "    'relation': 'opera.opera_production.designers',\n",
       "    'friendly_name': 'Designers'}]},\n",
       " 'sparql_query': 'PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX : <http://rdf.freebase.com/ns/> \\nSELECT (?x0 AS ?value) WHERE {\\nSELECT DISTINCT ?x0  WHERE { \\n?x0 :type.object.type :opera.opera_designer_role . \\n?x1 :type.object.type :opera.opera_designer_gig . \\nVALUES ?x2 { :m.0pm2fgf } \\n?x1 :opera.opera_designer_gig.design_role ?x0 . \\n?x2 :opera.opera_production.designers ?x1 . \\nFILTER ( ?x0 != ?x1 && ?x0 != ?x2 && ?x1 != ?x2  )\\n}\\n}',\n",
       " 'domains': ['opera'],\n",
       " 'level': 'i.i.d.',\n",
       " 's_expression': '(AND opera.opera_designer_role (JOIN (R opera.opera_designer_gig.design_role) (JOIN (R opera.opera_production.designers) m.0pm2fgf)))'}"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 87;\n",
       "                var nbb_unformatted_code = \"devs[0]\";\n",
       "                var nbb_formatted_code = \"devs[0]\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "devs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02c20e3f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "828b886f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21d9bea2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87b2d0cc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "377px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
